{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "FLAGS = None\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们在这里调用系统提供的Mnist数据函数为我们读入数据，如果没有下载的话则进行下载。\n",
    "\n",
    "<font color=#ff0000>**数据存放目录为 ./input_data**</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./input_data\\train-images-idx3-ubyte.gz\n",
      "Extracting ./input_data\\train-labels-idx1-ubyte.gz\n",
      "Extracting ./input_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting ./input_data\\t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "data_dir = './input_data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "一个一般简陋的模型，有两层隐层的神经网络"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the model\n",
    "def get_w_b(shape, stdev, lamda):\n",
    "    \"\"\"获取weight和biase，并添加L2正则\"\"\"\n",
    "    w = tf.Variable(tf.truncated_normal(shape, stddev=stdev))\n",
    "    b = tf.Variable(tf.zeros([shape[1]]))\n",
    "    tf.add_to_collection(\"losses\", tf.contrib.layers.l2_regularizer(lamda)(w))\n",
    "    return w,b\n",
    "\n",
    "w_stdev = 0.1\n",
    "loss_lamda = 0.0001\n",
    "net_array = [784, 500, 500] #第一个为输出层，后两个为隐层\n",
    "len_array = len(net_array)\n",
    "net_out = 10\n",
    "\n",
    "x = tf.placeholder(tf.float32, [None, net_array[0]])\n",
    "\n",
    "curr_y = x\n",
    "\n",
    "in_w = net_array[0]\n",
    "for i in range(len_array-1):\n",
    "    i = i+1\n",
    "    out_w = net_array[i]\n",
    "    w,b = get_w_b([in_w, out_w], w_stdev, loss_lamda)\n",
    "    curr_y = tf.nn.relu(tf.matmul(curr_y, w) + b) #采用relu激励\n",
    "    in_w = out_w\n",
    "\n",
    "w,b = get_w_b([net_array[-1], net_out], w_stdev, loss_lamda)\n",
    "y = tf.matmul(curr_y, w) + b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "定义我们的ground truth 占位符"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define loss and optimizer\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接下来我们计算交叉熵，注意这里不要使用注释中的手动计算方式，而是使用系统函数。\n",
    "另一个注意点就是，softmax_cross_entropy_with_logits的logits参数是**未经激活的wx+b**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The raw formulation of cross-entropy,\n",
    "#\n",
    "#   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),\n",
    "#                                 reduction_indices=[1]))\n",
    "#\n",
    "# can be numerically unstable.\n",
    "#\n",
    "# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw\n",
    "# outputs of 'y', and then average across the batch.\n",
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y)) \\\n",
    "                + tf.add_n(tf.get_collection(\"losses\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "生成一个训练step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "global_step = tf.Variable(0)  \n",
    "learning_rate = tf.train.exponential_decay(0.5, global_step, 100, 0.98, staircase=False) #设置学习率，初始为0.5，每过100次衰减为0.98\n",
    "\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(cross_entropy, global_step=global_step)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在这里我们仍然调用系统提供的读取数据，为我们取得一个batch。\n",
    "然后我们运行3k个step(5 epochs)，对权重进行优化。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "times:  0 , loss= 4.12058 , learning rate= 0.499899 , H= 0.0958\n",
      "times:  500 , loss= 0.519402 , learning rate= 0.451869 , H= 0.9544\n",
      "times:  1000 , loss= 0.335922 , learning rate= 0.408454 , H= 0.9733\n",
      "times:  1500 , loss= 0.375883 , learning rate= 0.36921 , H= 0.9658\n",
      "times:  2000 , loss= 0.278377 , learning rate= 0.333737 , H= 0.975\n",
      "times:  2500 , loss= 0.279084 , learning rate= 0.301672 , H= 0.9775\n",
      "times:  3000 , loss= 0.247579 , learning rate= 0.272687 , H= 0.9794\n",
      "times:  3500 , loss= 0.228841 , learning rate= 0.246488 , H= 0.9799\n",
      "times:  4000 , loss= 0.223604 , learning rate= 0.222805 , H= 0.9815\n",
      "times:  4500 , loss= 0.220441 , learning rate= 0.201398 , H= 0.9816\n",
      "times:  5000 , loss= 0.217835 , learning rate= 0.182048 , H= 0.9817\n",
      "times:  5500 , loss= 0.211102 , learning rate= 0.164557 , H= 0.9818\n",
      "times:  6000 , loss= 0.217499 , learning rate= 0.148747 , H= 0.9821\n",
      "times:  6500 , loss= 0.207059 , learning rate= 0.134455 , H= 0.982\n",
      "times:  7000 , loss= 0.200699 , learning rate= 0.121537 , H= 0.9825\n",
      "times:  7500 , loss= 0.199094 , learning rate= 0.10986 , H= 0.9817\n",
      "times:  8000 , loss= 0.199013 , learning rate= 0.0993045 , H= 0.9823\n",
      "times:  8500 , loss= 0.196794 , learning rate= 0.0897634 , H= 0.9822\n",
      "times:  9000 , loss= 0.197388 , learning rate= 0.081139 , H= 0.9814\n",
      "times:  9500 , loss= 0.195917 , learning rate= 0.0733433 , H= 0.9817\n",
      "times:  10000 , loss= 0.193031 , learning rate= 0.0662965 , H= 0.9822\n",
      "times:  10500 , loss= 0.190817 , learning rate= 0.0599268 , H= 0.9818\n",
      "times:  11000 , loss= 0.190029 , learning rate= 0.0541691 , H= 0.9816\n",
      "times:  11500 , loss= 0.188434 , learning rate= 0.0489646 , H= 0.9821\n",
      "times:  12000 , loss= 0.187372 , learning rate= 0.0442601 , H= 0.9823\n",
      "times:  12500 , loss= 0.188894 , learning rate= 0.0400076 , H= 0.9821\n",
      "times:  13000 , loss= 0.188531 , learning rate= 0.0361637 , H= 0.9821\n",
      "times:  13500 , loss= 0.188488 , learning rate= 0.0326892 , H= 0.9822\n",
      "times:  14000 , loss= 0.185107 , learning rate= 0.0295484 , H= 0.9821\n",
      "times:  14500 , loss= 0.189687 , learning rate= 0.0267094 , H= 0.9822\n",
      "times:  15000 , loss= 0.188184 , learning rate= 0.0241432 , H= 0.9822\n"
     ]
    }
   ],
   "source": [
    "# Train\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "for _ in range(15001):\n",
    "    batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "    iii, loss = sess.run([train_step, cross_entropy], feed_dict={x: batch_xs, y_: batch_ys})\n",
    "    if 0 == _%500:\n",
    "        H = sess.run(accuracy, feed_dict={x: mnist.test.images, y_: mnist.test.labels})\n",
    "        rate = sess.run(learning_rate)\n",
    "        print('times: ', _, ', loss=', loss, ', learning rate=', rate, ', H=', H)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "验证我们模型在测试数据上的准确率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9822\n"
     ]
    }
   ],
   "source": [
    "# Test trained model\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                      y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "毫无疑问，这个模型是一个非常简陋，性能也不理想的模型。目前只能达到92%左右的准确率。\n",
    "接下来，希望大家利用现有的知识，将这个模型优化至98%以上的准确率。\n",
    "Hint：\n",
    "- 多隐层\n",
    "- 激活函数\n",
    "- 正则化\n",
    "- 初始化\n",
    "- 摸索一下各个超参数\n",
    "  - 隐层神经元数量\n",
    "  - 学习率\n",
    "  - 正则化惩罚因子\n",
    "  - 最好每隔几个step就对loss、accuracy等等进行一次输出，这样才能有根据地进行调整"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
